# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from datetime import datetime
from spiderproject.items import ZhilianItem
import re
class ZhilianSpider(CrawlSpider):
    name = 'zhilian'
    allowed_domains = ['zhaopin.com','highpin.cn']
    start_urls = ['https://www.zhaopin.com/']
    custom_settings = {
        'ITEM_PIPELINES' : {
           'spiderproject.pipelines.SpiderprojectPipeline': 1,
        },
    'CONCURRENT_REQUESTS': 50
    }


    rules = (
        #城市
        Rule(LinkExtractor(allow=(r'/citymap') ,tags=('a',) ,attrs=('href',)) ,follow=True),
        # 校园xiaoyuan.zhaopin
        Rule(LinkExtractor(allow=(r'xiaoyuan\.zhaopin.com$'), tags=('a',), attrs=('href',)),  follow=True),
        #高端职位highpin.cn
        Rule(LinkExtractor(allow=(r'www.highpin.cn/'), tags=('a',), attrs=('href',)),  follow=True),
        #搜索sou.zhaopin.com
        Rule(LinkExtractor(allow=(r'sou.zhaopin.com'), tags=('a',), attrs=('href',)), follow=True),
        #http://company.zhaopin.com/CZ697981580.htm
        Rule(LinkExtractor(allow=(r'company\.zhaopin\.com/.+htm'), tags=('a',), attrs=('href',)), follow=True),

        #详情https://jobs.zhaopin.com/CC134218700J00114011807.htm
        Rule(LinkExtractor(allow=(r'jobs\.zhaopin\.com/.+htm'), tags=('a',), attrs=('href',)), callback='parse_item',follow=True),
    )

    def parse_item(self, response):
        item = ZhilianItem()
        url = response.url
        title = response.css('div.inner-left h1::text').extract()[0]
        smoney, emoney =self.edit_gongzi(response.xpath('//ul[@class="terminal-ul clearfix"]//li[1]/strong/text()').extract()[0])
        try:
            city = response.xpath('//ul[@class="terminal-ul clearfix"]//li[2]/strong/text()').extract()[0]
        except:
            city = response.xpath('//ul[@class="terminal-ul clearfix"]//li[2]/strong/a/text()').extract()[0]
        release_time =response.xpath('//ul[@class="terminal-ul clearfix"]//li[3]/strong/span/text()').extract()[0]
        job_type = response.xpath('//ul[@class="terminal-ul clearfix"]//li[4]/strong/text()').extract()[0]
        syear, eyear =self.edit_date( response.xpath('//ul[@class="terminal-ul clearfix"]//li[5]/strong/text()').extract()[0])
        edu = response.xpath('//ul[@class="terminal-ul clearfix"]//li[6]/strong/text()').extract()[0]
        group = response.xpath('//ul[@class="terminal-ul clearfix"]//li[8]/strong/a/text()').extract()[0]
        content = ','.join((response.xpath('//div[@class="tab-inner-cont"][1]//p//text()').extract()))
        adds = response.xpath('//div[@class="tab-inner-cont"][1]//h2//text()').extract()[0]
        label= self.joint(response.xpath('//div[@class="welfare-tab-box"][1]//span//text()').extract())

        item["url"] = url
        item["title"] = title
        item["smoney"] = smoney
        item["emoney"] = emoney
        item["city"] = city
        item["release_time"] = release_time
        item["job_type"] = job_type
        item["syear"] = syear
        item["eyear"] = eyear
        item["edu"] = edu
        item["groups"] = group
        item["content"] = content
        item["adds"] = adds
        item["label"] = label
        yield item

    #拼接列表
    def joint(self,value):
        v='-'.join(value)
        return v

    #取数字
    def number(self,value):
        reg = re.compile(r'(\d+)')
        res = reg.findall(value)
        return res

    #编辑工资
    def edit_gongzi(self,value):
        if '元/月'in value:
            res = self.number(value)
            smoney,emoney = int(res[0]),int(res[1])
            return smoney,emoney
        elif '面议' in value:
            smoney, emoney =9999, 99999
        return smoney, emoney

    #编辑工作年限
    def edit_date(self, value):
        num_reg = re.compile(r'\d+')
        if '-' in value:
            res =(num_reg.findall(value))
            syear,eyear =int(res[0]),int(res[1])
        elif '以下' in value:
            res = num_reg.search(value).group()
            syear, eyear = int(res), int(res)

        elif "不限" in value:
            syear, eyear = 0, 0
        return syear, eyear












